package org.weso.acota.utils;

import java.io.Reader;
import org.apache.log4j.Logger;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LengthFilter;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.PorterStemFilter;
import org.apache.lucene.analysis.StopAnalyzer;
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.Version;

public class EnglishStopAnalyzer extends Analyzer {
	protected static Logger logger = Logger.getLogger(EnglishStopAnalyzer.class);

	
	public EnglishStopAnalyzer() {
	}

	@SuppressWarnings("deprecation")
	@Override
	public TokenStream tokenStream(String arg0, Reader reader) {
		//Quitamos signos de puntuación
		logger.debug("Remove white spaces and punctuations marks");
		TokenStream result = new StandardTokenizer(Version.LUCENE_31, reader);  
        
		//change to lower case
		logger.debug("Change to lower case");
		result = new LowerCaseFilter(result);
		
		//remove word by length
		logger.debug("Remove word with less than o greather than 50 in length");
		result = new LengthFilter(result, 3, 50);
		
		//remove the stop words
		logger.debug("Remove stop words");
		result = new StopFilter(Version.LUCENE_CURRENT,result, StopAnalyzer.ENGLISH_STOP_WORDS_SET); 
		
		logger.debug("Remove word with less than o greather than 50 in length");
		result = new PorterStemFilter(result);
		
		//logger.info("Get stemmer of each words");
		//result = new SpanishStemFilter(result); 
		
		return result;
	}

}
